{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-23T14:20:08.790870Z",
     "start_time": "2019-10-23T14:19:58.596074Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "matchzoo version 1.0\n",
      "`ranking_task` initialized with metrics [normalized_discounted_cumulative_gain@3(0.0), normalized_discounted_cumulative_gain@5(0.0), mean_average_precision(0.0)]\n",
      "data loading ...\n",
      "data loaded as `train_pack_raw` `dev_pack_raw` `test_pack_raw`\n"
     ]
    }
   ],
   "source": [
    "%run init.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-23T14:10:06.000639Z",
     "start_time": "2019-10-23T14:10:05.995651Z"
    }
   },
   "outputs": [],
   "source": [
    "preprocessor = mz.models.MatchSRNN.get_default_preprocessor()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-23T14:10:30.199968Z",
     "start_time": "2019-10-23T14:10:06.004631Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing text_left with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 2118/2118 [00:00<00:00, 7281.82it/s]\n",
      "Processing text_right with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 18841/18841 [00:04<00:00, 4522.13it/s]\n",
      "Processing text_right with append: 100%|██████████| 18841/18841 [00:00<00:00, 1119126.53it/s]\n",
      "Building FrequencyFilter from a datapack.: 100%|██████████| 18841/18841 [00:00<00:00, 176766.51it/s]\n",
      "Processing text_right with transform: 100%|██████████| 18841/18841 [00:00<00:00, 186825.35it/s]\n",
      "Processing text_left with extend: 100%|██████████| 2118/2118 [00:00<00:00, 839653.67it/s]\n",
      "Processing text_right with extend: 100%|██████████| 18841/18841 [00:00<00:00, 817140.92it/s]\n",
      "Building Vocabulary from a datapack.: 100%|██████████| 418401/418401 [00:00<00:00, 3358771.46it/s]\n",
      "Processing text_left with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 2118/2118 [00:00<00:00, 10566.69it/s]\n",
      "Processing text_right with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 18841/18841 [00:03<00:00, 5337.62it/s]\n",
      "Processing text_right with transform: 100%|██████████| 18841/18841 [00:00<00:00, 146227.29it/s]\n",
      "Processing text_left with transform: 100%|██████████| 2118/2118 [00:00<00:00, 237915.74it/s]\n",
      "Processing text_right with transform: 100%|██████████| 18841/18841 [00:00<00:00, 128047.47it/s]\n",
      "Processing length_left with len: 100%|██████████| 2118/2118 [00:00<00:00, 805689.81it/s]\n",
      "Processing length_right with len: 100%|██████████| 18841/18841 [00:00<00:00, 838264.61it/s]\n",
      "Processing text_left with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 122/122 [00:00<00:00, 8968.00it/s]\n",
      "Processing text_right with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 1115/1115 [00:00<00:00, 5706.29it/s]\n",
      "Processing text_right with transform: 100%|██████████| 1115/1115 [00:00<00:00, 160472.46it/s]\n",
      "Processing text_left with transform: 100%|██████████| 122/122 [00:00<00:00, 140039.71it/s]\n",
      "Processing text_right with transform: 100%|██████████| 1115/1115 [00:00<00:00, 148257.96it/s]\n",
      "Processing length_left with len: 100%|██████████| 122/122 [00:00<00:00, 242055.39it/s]\n",
      "Processing length_right with len: 100%|██████████| 1115/1115 [00:00<00:00, 743860.18it/s]\n",
      "Processing text_left with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 237/237 [00:00<00:00, 10727.47it/s]\n",
      "Processing text_right with chain_transform of Tokenize => Lowercase => PuncRemoval: 100%|██████████| 2300/2300 [00:00<00:00, 5637.60it/s]\n",
      "Processing text_right with transform: 100%|██████████| 2300/2300 [00:00<00:00, 141228.56it/s]\n",
      "Processing text_left with transform: 100%|██████████| 237/237 [00:00<00:00, 166173.53it/s]\n",
      "Processing text_right with transform: 100%|██████████| 2300/2300 [00:00<00:00, 19426.52it/s]\n",
      "Processing length_left with len: 100%|██████████| 237/237 [00:00<00:00, 155880.52it/s]\n",
      "Processing length_right with len: 100%|██████████| 2300/2300 [00:00<00:00, 388893.78it/s]\n"
     ]
    }
   ],
   "source": [
    "train_pack_processed = preprocessor.fit_transform(train_pack_raw)\n",
    "dev_pack_processed = preprocessor.transform(dev_pack_raw)\n",
    "test_pack_processed = preprocessor.transform(test_pack_raw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-23T14:10:30.232878Z",
     "start_time": "2019-10-23T14:10:30.202958Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'filter_unit': <matchzoo.preprocessors.units.frequency_filter.FrequencyFilter at 0x7f1042783710>,\n",
       " 'vocab_unit': <matchzoo.preprocessors.units.vocabulary.Vocabulary at 0x7f10503df400>,\n",
       " 'vocab_size': 30058,\n",
       " 'embedding_input_dim': 30058}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocessor.context"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-23T14:10:38.899753Z",
     "start_time": "2019-10-23T14:10:30.236869Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "glove_embedding = mz.datasets.embeddings.load_glove_embedding(dimension=100)\n",
    "term_index = preprocessor.context['vocab_unit'].state['term_index']\n",
    "embedding_matrix = glove_embedding.build_matrix(term_index)\n",
    "l2_norm = np.sqrt((embedding_matrix * embedding_matrix).sum(axis=1))\n",
    "embedding_matrix = embedding_matrix / l2_norm[:, np.newaxis]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-23T14:10:38.901741Z",
     "start_time": "2019-10-23T14:09:48.855Z"
    }
   },
   "outputs": [],
   "source": [
    "trainset = mz.dataloader.Dataset(\n",
    "    data_pack=train_pack_processed,\n",
    "    mode='pair',\n",
    "    num_dup=2,\n",
    "    num_neg=1\n",
    ")\n",
    "testset = mz.dataloader.Dataset(\n",
    "    data_pack=test_pack_processed\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-23T14:10:38.904703Z",
     "start_time": "2019-10-23T14:09:48.860Z"
    }
   },
   "outputs": [],
   "source": [
    "padding_callback = mz.models.MatchSRNN.get_default_padding_callback()\n",
    "\n",
    "trainloader = mz.dataloader.DataLoader(\n",
    "    dataset=trainset,\n",
    "    batch_size=20,\n",
    "    stage='train',\n",
    "    resample=True,\n",
    "    sort=False,\n",
    "    callback=padding_callback\n",
    ")\n",
    "testloader = mz.dataloader.DataLoader(\n",
    "    dataset=testset,\n",
    "    batch_size=20,\n",
    "    stage='dev',\n",
    "    callback=padding_callback\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-23T14:10:38.906698Z",
     "start_time": "2019-10-23T14:09:48.864Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MatchSRNN(\n",
      "  (embedding): Embedding(30058, 100)\n",
      "  (dropout): Dropout(p=0.2, inplace=False)\n",
      "  (matching_tensor_network): MatchingTensor()\n",
      "  (spatial_gru_network): SpatialGRU(\n",
      "    (_wr): Linear(in_features=34, out_features=30, bias=True)\n",
      "    (_wz): Linear(in_features=34, out_features=40, bias=True)\n",
      "    (_w_ij): Linear(in_features=4, out_features=10, bias=True)\n",
      "    (_U): Linear(in_features=30, out_features=10, bias=False)\n",
      "  )\n",
      "  (out): Linear(in_features=10, out_features=1, bias=True)\n",
      ")\n",
      "Trainable params:  3048611\n"
     ]
    }
   ],
   "source": [
    "model = mz.models.MatchSRNN()\n",
    "\n",
    "model.params['task'] = ranking_task\n",
    "model.params['embedding'] = embedding_matrix\n",
    "model.params['channels'] = 4\n",
    "model.params['units'] = 10\n",
    "model.params['dropout'] = 0.2\n",
    "model.params['direction'] = 'lt'\n",
    "\n",
    "model.build()\n",
    "\n",
    "print(model)\n",
    "print('Trainable params: ', sum(p.numel() for p in model.parameters() if p.requires_grad))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-23T14:10:38.909692Z",
     "start_time": "2019-10-23T14:09:48.867Z"
    }
   },
   "outputs": [],
   "source": [
    "optimizer = torch.optim.Adadelta(model.parameters())\n",
    "\n",
    "trainer = mz.trainers.Trainer(\n",
    "    model=model,\n",
    "    optimizer=optimizer,\n",
    "    trainloader=trainloader,\n",
    "    validloader=testloader,\n",
    "    validate_interval=None,\n",
    "    epochs=10\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-10-23T14:10:38.911685Z",
     "start_time": "2019-10-23T14:09:48.872Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "59946629514b4d3995c47969c63bc06b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=102), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Iter-102 Loss-0.994]:\n",
      "  Validation: normalized_discounted_cumulative_gain@3(0.0): 0.356 - normalized_discounted_cumulative_gain@5(0.0): 0.4454 - mean_average_precision(0.0): 0.414\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2a710b084df9443fbf2999a926c954fa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=102), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Iter-204 Loss-0.740]:\n",
      "  Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5853 - normalized_discounted_cumulative_gain@5(0.0): 0.6469 - mean_average_precision(0.0): 0.6044\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cf90e64aeefe4699b9b150f67efa03a1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=102), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Iter-306 Loss-0.544]:\n",
      "  Validation: normalized_discounted_cumulative_gain@3(0.0): 0.6018 - normalized_discounted_cumulative_gain@5(0.0): 0.6674 - mean_average_precision(0.0): 0.6209\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "503d804892cf4f02836db28c136a0a10",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=102), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Iter-408 Loss-0.421]:\n",
      "  Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5979 - normalized_discounted_cumulative_gain@5(0.0): 0.664 - mean_average_precision(0.0): 0.6192\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "342b097c46e94faba09241fc9f384e8e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=102), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Iter-510 Loss-0.303]:\n",
      "  Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5653 - normalized_discounted_cumulative_gain@5(0.0): 0.63 - mean_average_precision(0.0): 0.586\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "540e8af1944b49c8a8e34b88cd3e9102",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=102), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Iter-612 Loss-0.229]:\n",
      "  Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5929 - normalized_discounted_cumulative_gain@5(0.0): 0.6377 - mean_average_precision(0.0): 0.5971\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fd634d76be2e4004971f3425ce92f616",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=102), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Iter-714 Loss-0.174]:\n",
      "  Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5513 - normalized_discounted_cumulative_gain@5(0.0): 0.6166 - mean_average_precision(0.0): 0.5796\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3ff326a25ead4fd8b6ae6279b95d4681",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=102), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Iter-816 Loss-0.117]:\n",
      "  Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5744 - normalized_discounted_cumulative_gain@5(0.0): 0.6258 - mean_average_precision(0.0): 0.5884\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cc298ed4ab824aceae50e486ac1fc158",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=102), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Iter-918 Loss-0.084]:\n",
      "  Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5467 - normalized_discounted_cumulative_gain@5(0.0): 0.6106 - mean_average_precision(0.0): 0.5672\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c3c495798577437abd92e30bab6fc9c8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=102), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Iter-1020 Loss-0.090]:\n",
      "  Validation: normalized_discounted_cumulative_gain@3(0.0): 0.5404 - normalized_discounted_cumulative_gain@5(0.0): 0.6036 - mean_average_precision(0.0): 0.558\n",
      "\n",
      "Cost time: 4545.272604942322s\n"
     ]
    }
   ],
   "source": [
    "trainer.run()"
   ]
  }
 ],
 "metadata": {
  "file_extension": ".py",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "mimetype": "text/x-python",
  "name": "python",
  "npconvert_exporter": "python",
  "pygments_lexer": "ipython3",
  "version": 3
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
